
import feapder
from feapder.utils import tools
import os
from feapder.utils import metrics
from feapder.network import user_agent
import random
from requests.models import Response as RResponse
import requests_go as ssl_requests
import platform
from feapder.db.mysqldb import MysqlDB

platform_os = platform.system()


def get_proxies():
    PROXIES_VN = [f'http://astoip753-country-VN-refreshMinutes-3:f58a5c-d41239-9a79b6-1556c4-cb7f76@38.84.70.122:1000{_}'
                  for _ in
                  range(1, 6)]
    # 第二批 3000
    P2 = [f'http://astoip753-country-US-refreshMinutes-3:f58a5c-d41239-9a79b6-1556c4-cb7f76@38.84.70.122:1000{_}' for _ in
          range(6, 10)]
    P3 = [f'http://astoip753-country-US-refreshMinutes-3:f58a5c-d41239-9a79b6-1556c4-cb7f76@38.84.70.122:100{_}' for _ in
          range(10, 26)]

    PROXIES = P2 + P3+PROXIES_VN
    ip = random.choice(PROXIES)
    proxies = {
        'http': ip,
        'https': ip,
    }
    # TODO 
    #     from utils.redis_utils import RedisClient
    # PROXIES = "tiktok:proxies"
    # IPS = RedisClient.get(PROXIES).split(",")
    # IP = random.choice(IPS)
    # proxies = {
    #     "http": f"socks5h://dsman:dsman@{IP}:8119",
    #     "https": f"socks5h://dsman:dsman@{IP}:8119",
    # }
    return proxies

class BaseSpider(feapder.BatchSpider):
    host = "https://www.amazon.com"  # 补充相对url
    seed_table = "amazon_task_seeds"

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.db = MysqlDB()

    def start_callback(self):
        print(f"爬虫:{type(self).__name__} 开始了")
        metrics.init()

    def end_callback(self):
        print(f"爬虫:{type(self).__name__} 结束了")
        metrics.close()

    def download_midware(self, request):
        print('使用自定义下载器', request.url)
        seed_md5_file_name = tools.get_md5(request.url)+".html"
        file_path = os.path.join("files", seed_md5_file_name)
        for i in range(2):

            if platform_os == 'Windows' and tools.is_exist(file_path):  # 只有是本地进行缓存文件
                content = tools.read_file(file_path).encode()
                print("file_path len  content", len(content), file_path)

                if b"Enter the characters you see below" in content or len(content) < 70:
                    continue
                response = RResponse()
                # 设置响应的属性
                response.status_code = 200
                response._content = content
                response.headers['Content-Type'] = 'text/html;charset=UTF-8'
                response.encoding = "utf-8"
                response = feapder.Response(response, False)
                return request,  response
        impersonate = random.choice(["edge99", "edge101", "chrome99", "chrome100",
                                    "chrome101", "chrome104", "chrome107", "chrome110", "safari15_3", "safari15_5"])

        default_headers = {
            'Host': 'www.amazon.com',
            # Requests sorts cookies= alphabetically
            'Cookie': 'aws-target-data=%7B%22support%22%3A%221%22%7D; session-id=141-2038706-9570445; i18n-prefs=USD; ubid-main=132-7922229-0312612; aws-target-visitor-id=1682529305907-332700.42_0; aws-ubid-main=588-3545106-0851264; AMCV_7742037254C95E840A4C98A6%40AdobeOrg=1585540135%7CMCIDTS%7C19821%7CMCMID%7C20017007120206226960042027017711098619%7CMCAAMLH-1713084953%7C11%7CMCAAMB-1713084953%7CRKhpRz8krg2tLO6pguXWp5olkAcUniQYPHaMWWgdJ3xzPWQmdj0y%7CMCOPTOUT-1712487353s%7CNONE%7CMCAID%7CNONE%7CvVersion%7C4.4.0; regStatus=registered; aws-userInfo=%7B%22arn%22%3A%22arn%3Aaws%3Aiam%3A%3A699515121180%3Aroot%22%2C%22alias%22%3A%22%22%2C%22username%22%3A%22AWS%22%2C%22keybase%22%3A%22%22%2C%22issuer%22%3A%22http%3A%2F%2Fsignin.aws.amazon.com%2Fsignin%22%2C%22signinType%22%3A%22PUBLIC%22%7D; lc-main=en_US; x-amz-captcha-1=1714048993374418; x-amz-captcha-2=omrUz/TOs92r0ouEkqRpBQ==; session-id-time=2082787201l; JSESSIONID=182A9C5265F9325F3EC845237DBF130A; session-token=sOI4nj0E5aJZxoU5hifqQJ6zDjvxbJ4i75nRH+xLvZSp/A4dx3CtzVie3VN6JrP/NUVm87x9qJMBElsjMku0l9zbJm9vw+66U49Xhda9R83hHGT6Es+s3UWQZqO5Pg4W02rGA4j1RufTKiSPDkj1/5WxM+LbrT7Kt/zRh/GhJTCW+fm0BJbui+PSKhRc219mu6NRnLNfaaaeTWIPkcI8bcZtnSMwiHNdtACzSk6JzaM9zNMNCl1e1b5/dA0mAvRPBHnKIa0rz0FxTqAT/a4EbfAvf+5xvdmcl6UTAoqRYF11UlDjyZB+7etS8GucFMih9JRSTqQBgxdui3IRKQoF8uRwy3532LfF; csm-hit=tb:MDH1C3HTJQ857NY2XV9D+s-7RQXGGAZTXJC3Z1732XV|1714372686520&t:1714372686520&adb:adblk_no',
            'cache-control': 'max-age=0',
            'device-memory': '8',
            'sec-ch-device-memory': '8',
            'dpr': '1.25',
            'sec-ch-dpr': '1.25',
            'viewport-width': '1536',
            'sec-ch-viewport-width': '1536',
            'rtt': '250',
            'downlink': '10',
            'ect': '4g',
            'sec-ch-ua': f'"Chromium";v="124", "{impersonate}";v="124", "Not-A.Brand";v="99"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"Windows"',
            'sec-ch-ua-platform-version': '"10.0.0"',
            'upgrade-insecure-requests': '1',
            # 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
            'user-agent': user_agent.get("chrome"),
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
            'sec-fetch-site': 'none',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-user': '?1',
            'sec-fetch-dest': 'document',
            'accept-language': 'en;q=0.9',
            'priority': 'u=0, i',
        }
        params = request.params if hasattr(request, "params") else None
        timeout = request.timeout if hasattr(request, "timeout") else None
        data = request.data if hasattr(request, "data") else None
        headers = default_headers
        headers.update(request.get_headers())
        method = request.method if request.method else 'GET'
        if 'data' in request.__dict__:
            method = "POST"
        if hasattr(request, "referer"):
            headers.update({'referer': request.referer})
        proxies = {'https': 'http://localhost:8880'}  if platform_os == 'Windows' else get_proxies()
        origin_response = ssl_requests.request(
            method=method, url=request.url, headers=headers, params=params, timeout=timeout, data=data, proxies=proxies, verify=False)
        origin_response.__dict__['_content'] = origin_response.content
        # response = requests.get(url = request.url, headers=headers, params=params, timeout=timeout,data=data,proxies=proxies)
        response = feapder.Response(origin_response, make_absolute_links=True)
        return request,  response

    def validate(self, request, response):
        if not response:
            metrics.emit_counter("response_none", count=1,
                                 classify=type(self).__name__)
            raise Exception("response is None")  # 重试
        if response.status_code != 200:   # 429  Request was throttled. Please wait a moment and refresh the page
            metrics.emit_counter(response.status_code,
                                 count=1, classify=type(self).__name__)
            raise Exception("response code not 200")  # 重试
        if response.content and "Enter the characters you see below" in response.text:
            metrics.emit_counter("characters", count=1,
                                 classify=type(self).__name__)
            raise Exception("验证码")
        if response.content and platform_os == 'Windows':
            seed_md5_file_name = tools.get_md5(request.url)+".html"
            file_path = os.path.join("files", seed_md5_file_name)
            print("save", file_path)
            tools.write_file(filename=file_path, content=response.text)

    def failed_request(self, request, response):
        """
        @summary: 超过最大重试次数的request
        ---------
        @param request:
        ---------
        @result: request / item / callback / None (返回值必须可迭代)
        """
        metrics.emit_counter("fail", count=1, classify=type(self).__name__)

        yield request
        yield self.update_task_state(request.task_id, -1)  # 更新任务状态为-1

    def exception_request(self, request, response):
        """
        @summary: 请求或者parser里解析出异常的request
        ---------
        @param request:
        @param response:
        ---------
        @result: request / callback / None (返回值必须可迭代)
        """

    def insert_asin(self, asin, cid):
        if not self.db.find(f"select asin from {self.seed_table} where asin='{asin}' limit 1"):
            self.db.add(
                f"insert into {self.seed_table} (asin,cid) values ('{asin}','{cid}') ")
